package com.gxljc.bear.crawler.itaogao.yangcheng;

import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.ObjectUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;

import com.alibaba.fastjson.JSONObject;
import com.gxljc.commons.util.Pair;
import com.gxljc.commons.util.StringUtil;
import com.gxljc.bear.crawler.itaogao.NewsPageTable;
import com.gxljc.bear.crawler.itaogao.util.DayUtil;
import com.gxljc.bear.crawler.itaogao.util.FetchDataUtil;
//import com.gxljc.bear.crawler.itaogao.util.HbaseUtil;
import com.gxljc.bear.crawler.itaogao.util.HttpUtil;
import com.gxljc.bear.crawler.itaogao.util.ImageUtil;
import com.gxljc.bear.crawler.itaogao.util.MongodbUtil;
import com.gxljc.bear.crawler.util.DateUtil;

/**
 * @author haitao E-mail:haitao@summba.com
 * @version createTime：2021年9月19日 上午9:40:40
 * 
 */

public class YangchengEveningNewsCrawler {

	public static final String YANGCHENG_URL_DEFAULT = "http://ep.ycwb.com/epaper/ycwb/html/";
	public static final String YANGCHENG_MARK = "../../../../";
	public static final String YANGCHENG_HEAD = "http://ep.ycwb.com/epaper/";
	private final static Logger LOG = Logger.getLogger(YangchengEveningNewsCrawler.class);
	private final static String SOURCE = "羊城晚报";
	private final static String HTTP_CHARSET = "UTF-8";
	private final static String MONGONAME_COLLECTION = "t_newspaper_yangcheng";
	private List<String> defaultSeeds = new ArrayList<String>();
	private List<String> detailSeeds = new ArrayList<String>();
	private List<String> articlSeeds = new ArrayList<String>();
	private Map<String, Object> newsMap = new HashMap<String, Object>();
	public static final String YANGCHENG_URL = "http://ep.ycwb.com/epaper/ycwb/";
	protected boolean proxy;
	private int perYear = 2016;// 默认2021年为起点
	private Calendar calendar;
	private int countNum = 0;
	protected String time;
	
	public void setProxy(Boolean proxy) {
		this.proxy = proxy;
	}
	
	public void setTime(String time) {
		this.time = time;
	}
	
	public YangchengEveningNewsCrawler() {
		newsMap.put("status", 0);
		newsMap.put("mediaName", SOURCE);
		calendar = Calendar.getInstance();
		perYear = calendar.get(Calendar.YEAR);
	}
	
	public void startCrawlerAll() {
		// 获取每一期的default页面
		try {
			LOG.info("=====start yangcheng evening news update crawler!=====");
			genDefaultSeeds();
			LOG.info(String.format("default url crawler finished! and default seeds size is %s", defaultSeeds.size()));
		} catch (Exception e) {
			e.printStackTrace();
			LOG.info("genDefaultSeeds functions Error!");
		}

		// 获取每一期板块页面
		genDetailSeeds();
		LOG.info(String.format("detail url crawler finished! and detail seeds size is %s", detailSeeds.size()));

		// 获取每一期每个板块的详细文章页面&&抓取存入数据库
		genArticleSeeds();
		LOG.info(String.format("All yangcheng crawler finished!,all crawler size is %s", countNum));

	}

	public void startCrawlerUpdate() {
		// 更新脚本只会抓取当天的新闻，前后都不在抓取，获取每一期的default页面
		try {
			LOG.info("=====start yangcheng evening news crawler!=====");
			genDefaultSeedsUpdate();
			LOG.info(String.format("default url crawler finished! and default seeds size is %s", defaultSeeds.size()));
		} catch (Exception e) {
			e.printStackTrace();
			LOG.info("genDefaultSeeds functions Error!");
		}
		
		
		// 获取每一期板块页面
		genDetailSeeds();
		LOG.info(String.format("detail url crawler finished! and detail seeds size is %s", detailSeeds.size()));

		// 获取每一期每个板块的详细文章页面&&抓取存入数据库
		genArticleSeeds();
		LOG.info(String.format("All yangcheng crawler finished!,all crawler size is %s", countNum));

	}
	
	public void startCrawlerUpdateSingle() {
		// 更新脚本只会抓取当天的新闻，前后都不在抓取，获取每一期的default页面
		genDefaultSeedsUpdateSingle(time);
		
		// 获取每一期板块页面
		genDetailSeeds();
		LOG.info(String.format("detail url crawler finished! and detail seeds size is %s", detailSeeds.size()));

		// 获取每一期每个板块的详细文章页面&&抓取存入数据库
		genArticleSeeds();
		LOG.info(String.format("All yangcheng crawler finished!,all crawler size is %s", countNum));

	}
	
	/**
	 * 获取新闻每一天的默认的种子url
	 */
	void genDefaultSeeds() {
		// 只抓当年的，比如今年2016，那么只抓2021年的，前后年都不抓
		for (int year = perYear; year <= perYear; year++) {
			for (int month = 1; month <= 12; month++) {
				for (int day = 1; day <= 31; day++) {
					String monthStr = DayUtil.handleValueLessThan10(month);
					String dayStr = DayUtil.handleValueLessThan10(day);
					String accessUrl = YANGCHENG_URL_DEFAULT + String.valueOf(year) + "-" + monthStr + "/" + dayStr
							+ "/";
					String defaultUrl = null;
					try {
						defaultUrl = HttpUtil.getHtmlContent(accessUrl, HTTP_CHARSET);
					} catch (Exception e) {
						LOG.info(String.format("get defaultUrl error! Error accessUrl is ", accessUrl));
						e.printStackTrace();
					}
					if (defaultUrl == null) {
						continue;
					}
					if (defaultUrl.contains("URL=")) {
						defaultUrl = accessUrl + defaultUrl.split("URL=")[1].split("\"")[0];
					} else {
						String splitStr = String.valueOf(year) + "-" + monthStr + "/" + dayStr + "/";
						defaultUrl = accessUrl + defaultUrl.split(splitStr)[1].split("\"")[0];
					}
					defaultSeeds.add(defaultUrl);
					LOG.info(String.format("defaultUrl is %s", defaultUrl));
				}
			}
		}
	}

	/**
	 * 获取新闻每一天的默认的种子url
	 */
	void genDefaultSeedsUpdate() {
		// 只抓当年的，比如今年2016，那么只抓2021年的，前后年都不抓
		Date day = new Date();
		SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
		String dateNowStr = sdf.format(day);
		String year = dateNowStr.split("-")[0];
		String monthStr = dateNowStr.split("-")[1];
		String dayStr = dateNowStr.split("-")[2];
		String accessUrl = YANGCHENG_URL_DEFAULT + year + "-" + monthStr + "/" + dayStr + "/";
		String defaultUrl = null;
		try {
			defaultUrl = HttpUtil.getHtmlContent(accessUrl, HTTP_CHARSET);
		} catch (Exception e) {
			LOG.info(String.format("get defaultUrl error! Error accessUrl is ", accessUrl));
			e.printStackTrace();
		}
		if (defaultUrl != null) {
			if (defaultUrl.contains("URL=")) {
				defaultUrl = accessUrl + defaultUrl.split("URL=")[1].split("\"")[0];
			} else {
				String splitStr = String.valueOf(year) + "-" + monthStr + "/" + dayStr + "/";
				defaultUrl = accessUrl + defaultUrl.split(splitStr)[1].split("\"")[0];
			}
			defaultSeeds.add(defaultUrl);
			LOG.info(String.format("defaultUrl is %s", defaultUrl));
		} else {
			LOG.info(String.format("defaultSeeds size is %s", defaultSeeds.size()));
		}
	}
	
	/**
	 * 获取新闻每一天的默认的种子url
	 */
	void genDefaultSeedsUpdateSingle(String time) {
		if(StringUtils.isBlank(time)){
			return;
		}
		String year = time.split("-")[0];
		String monthStr = time.split("-")[1];
		String dayStr = time.split("-")[2];
		String accessUrl = YANGCHENG_URL_DEFAULT + year + "-" + monthStr + "/" + dayStr + "/";
		String defaultUrl = null;
		try {
			defaultUrl = HttpUtil.getHtmlContent(accessUrl, HTTP_CHARSET);
		} catch (Exception e) {
			LOG.info(String.format("get defaultUrl error! Error accessUrl is ", accessUrl));
			e.printStackTrace();
		}
		if (defaultUrl != null) {
			if (defaultUrl.contains("URL=")) {
				defaultUrl = accessUrl + defaultUrl.split("URL=")[1].split("\"")[0];
			} else {
				String splitStr = String.valueOf(year) + "-" + monthStr + "/" + dayStr + "/";
				defaultUrl = accessUrl + defaultUrl.split(splitStr)[1].split("\"")[0];
			}
			defaultSeeds.add(defaultUrl);
			LOG.info(String.format("defaultUrl is %s", defaultUrl));
		} else {
			LOG.info(String.format("defaultSeeds size is %s", defaultSeeds.size()));
		}
	}
	
	
	@SuppressWarnings("unchecked")
	void genDetailSeeds() {
		if (defaultSeeds.size() == 0) {
			LOG.info(String.format("defaultSeeds Size is %s", defaultSeeds.size()));
			return;
		}
		for (String url : defaultSeeds) {
			Pair<String, byte[]> pair = FetchDataUtil.fetResult(url, proxy);
			if (pair == null) {
				// 尝试其他url获取pair值
				String[] urlSplit = url.split("node_");
				String urlFirst = urlSplit[0];
				int urlId = Integer.parseInt(urlSplit[1].replace(".htm", "")) + 1;
				String urlSecond = String.valueOf(urlId) + ".htm";
				pair = FetchDataUtil.fetResult(urlFirst + "node_" + urlSecond, proxy);
			}
			if (pair == null) {
				continue;
			}

			String result = "";
			result = pair.first;
			if (result != null) {
				JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
				if (jsonResult != null && jsonResult.get("sectionsUrl") != null) {
					for (String urlDetail : (List<String>) jsonResult.get("sectionsUrl")) {
						detailSeeds.add(urlDetail.replace(YANGCHENG_MARK, ""));
					}
				}
			}
		}

	}

	@SuppressWarnings("unchecked")
	void genArticleSeeds() {
		if (detailSeeds.size() == 0) {
			LOG.info(String.format("detailSeeds Size is %s", detailSeeds.size()));
			return;
		}
		for (String dtUrl : detailSeeds) {
			// 添加头部
			dtUrl = YANGCHENG_HEAD + dtUrl;
			Pair<String, byte[]> pair = FetchDataUtil.fetResult(dtUrl, proxy);
			if (pair == null) {
				continue;
			}
			String result = "";
			result = pair.first;
			if (result != null) {
				JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
				if (jsonResult != null && jsonResult.get("titleUrl") != null) {
					for (String contentUrl : (List<String>) jsonResult.get("titleUrl")) {
						String atUrl = dtUrl.split("node")[0] + contentUrl;
						if (!MongodbUtil.needReCrawl(atUrl, MONGONAME_COLLECTION)) {
							articlSeeds.add(atUrl);
						}
					}
				}
			}

			// 每5000个url，集中抓取一次
			if (articlSeeds.size() % 5000 == 0) {
				crawlerNews(articlSeeds);
				countNum += 5000;
				articlSeeds.clear();
			}
		}
		// 除不尽的 仍旧插入
		if (articlSeeds.size() > 0) {
			crawlerNews(articlSeeds);
			countNum += articlSeeds.size();
		}
	}

	@SuppressWarnings("unchecked")
	void crawlerNews(List<String> atUrl) {
		for (String url : atUrl) {
			Pair<String, byte[]> pair = FetchDataUtil.fetResult(url, proxy);
			String result = "";
			result = pair.first;
			byte[] html = pair.second;
			NewsPageTable table = new NewsPageTable();
			if (result != null) {
				JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
				// 来源指定为"羊城晚报"，非指定来源continue
				String source = ObjectUtils.toString(jsonResult.get("source"));
				if (source.contains("来源")) {
					try {
						source = source.split("来源:|来源：")[1];
					} catch (Exception e) {
						LOG.info(String.format("source is error %s", source));
					}
				}
			
				List<String> contentList = new ArrayList<String>();
				Object content = jsonResult.get("content");
				if (content != null) {
					contentList = (List<String>) content;
				}
				if (contentList.size() == 0) {
					continue;
				}
				String id = ObjectUtils.toString(StringUtil.genMD5Val(url));
				String title = ObjectUtils.toString(jsonResult.get("title"));
				String subTitle = ObjectUtils.toString(jsonResult.get("subTitle"));
				String logoUrl = ObjectUtils.toString(jsonResult.get("logoUrl"));
				String author = ObjectUtils.toString(jsonResult.get("author"));
				String publishDate = ObjectUtils.toString(jsonResult.get("publicTime"));
				Object images = jsonResult.get("images");
				List<String> imagesPlatform = new ArrayList<String>();
				if (images != null) {
					List<String> imagesList = (List<String>) images;
					try {
						imagesPlatform = ImageUtil.ImageUrlService(imagesList, url);
					} catch (Exception e) {
						LOG.info("images convert Error！");
						// e.printStackTrace();
					}
				}
				if (imagesPlatform.size() != 0) {
					newsMap.put("images", imagesPlatform);
				}
				if (author.split("作者：").length == 2) {
					author = author.split("作者：")[1].trim();
					if (author.equals("作者：") || author.equals("作者:")) {
						author = "";
					}
				}
				if (!logoUrl.isEmpty()) {
					try {
						logoUrl = logoUrl.replace("../../../", YANGCHENG_URL);
						logoUrl = ImageUtil.imageConvertplatform(logoUrl);
					} catch (Exception e) {
						LOG.info(String.format("logoUrl convert failed! logoUrl is %s", logoUrl));
						e.printStackTrace();
					}
				}

				// 添加基础数据
				newsMap.put("_id", id);
				newsMap.put("title", title);
				if (title.equals(subTitle)) {
					newsMap.put("subTitle", "");
				} else {
					newsMap.put("subTitle", subTitle);
				}
				Date pd = DateUtil.parseFormat(publishDate, "yyyy年MM月dd日");
				if (pd != null) {
					newsMap.put("publishDate", pd);
				}
				newsMap.put("logoUrl", logoUrl);
				newsMap.put("content", StringUtils.join(contentList, "####").replaceAll("　　####|　　", "").trim());
				newsMap.put("author", author);
				newsMap.put("crawlUrl", url);
				newsMap.put("crawlFlag", 1);
				newsMap.put("crawlDate", new Date());
				newsMap.put("source", source);

				// 更新mongdb
				MongodbUtil.update2Mongodb(newsMap, MONGONAME_COLLECTION);

				table.setTitle(title);
				table.setId(id);

				// 更新到hbase
				// HbaseUtil.saveHbase(table, html);

				LOG.info(String.format("insert the id/rowkey is %s ", id));

			}
		}
	}

	@SuppressWarnings("unchecked")
	void crawlerNewsSingle(String url) {
		Pair<String, byte[]> pair = FetchDataUtil.fetResult(url, proxy);
		if(pair == null){
			return;
		}
		String result = "";
		result = pair.first;
		byte[] html = pair.second;
		NewsPageTable table = new NewsPageTable();
		if (result != null) {
			JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
			String source = ObjectUtils.toString(jsonResult.get("source"));
			// 来源指定为"羊城晚报"，非指定来源continue
			List<String> contentList = new ArrayList<String>();
			Object content = jsonResult.get("content");
			if (content != null) {
				contentList = (List<String>) content;
			}
			if (contentList.size() == 0) {
				return;
			}
			String id = ObjectUtils.toString(StringUtil.genMD5Val(url));
			String title = ObjectUtils.toString(jsonResult.get("title"));
			String subTitle = ObjectUtils.toString(jsonResult.get("subTitle"));
			String logoUrl = ObjectUtils.toString(jsonResult.get("logoUrl"));
			String author = ObjectUtils.toString(jsonResult.get("author"));
			String publishDate = ObjectUtils.toString(jsonResult.get("publicTime"));
			Object images = jsonResult.get("images");
			List<String> imagesPlatform = new ArrayList<String>();
			if (images != null) {
				List<String> imagesList = (List<String>) images;
				try {
					imagesPlatform = ImageUtil.ImageUrlService(imagesList, url);
				} catch (Exception e) {
					LOG.info("images convert Error！");
				}
			}
			if (imagesPlatform.size() != 0) {
				newsMap.put("images", imagesPlatform);
			}

			if (author.split("作者：").length == 2) {
				author = author.split("作者：")[1].trim();
			}
			if (!logoUrl.isEmpty()) {
				try {
					logoUrl = logoUrl.replace("../../../", YANGCHENG_URL);
					logoUrl = ImageUtil.imageConvertplatform(logoUrl);
				} catch (Exception e) {
					LOG.info(String.format("logoUrl convert failed! logoUrl is %s", logoUrl));
					e.printStackTrace();
				}
			}

			// 添加基础数据
			newsMap.put("_id", id);
			newsMap.put("title", title);
			if (title.equals(subTitle)) {
				newsMap.put("subTitle", "");
			} else {
				newsMap.put("subTitle", subTitle);
			}
			Date pd = DateUtil.parseFormat(publishDate, "yyyy年MM月dd日");
			if (pd != null) {
				newsMap.put("publishDate", pd == null ? "" : pd);
			}
			newsMap.put("logoUrl", logoUrl);
			newsMap.put("content", StringUtils.join(contentList, "####").replaceAll("　　####|　　", "").trim());
			newsMap.put("author", author);
			newsMap.put("crawlUrl", url);
			newsMap.put("crawlFlag", 1);
			newsMap.put("crawlDate", new Date());
			newsMap.put("source", source);

			// 更新mongdb
			MongodbUtil.update2Mongodb(newsMap, MONGONAME_COLLECTION);

			table.setTitle(title);
			table.setId(id);

			// 更新到hbase
			// HbaseUtil.saveHbase(table, html);

			LOG.info(String.format("insert the id/rowkey is %s ", id));

		}
	}

}
